#!/usr/bin/env bash

# 验证集多 checkpoint 批量评估脚本
# 遍历 ${BASE_DIR}/saved/train/ 下所有名称包含 "checkpoint" 的目录作为 CKPT_DIR
# 对每个 checkpoint 运行一次 eval.py，subjects_split 使用 "val"
# eval.py 内部会自动在输出文件名后追加 ckpt 目录名后缀

set -euo pipefail
shopt -s nullglob

BASE_DIR=${BASE_DIR:-"/public/home/jianghaitenggroup/jianglab/WorkSpace/Qwen25_VL_FT"}
SYSTEM_PROMPT=${SYSTEM_PROMPT:-"${BASE_DIR}/system_prompt.md"}
SUBJECTS_JSON=${SUBJECTS_JSON:-"${BASE_DIR}/saved/data/subjects.json"}
IMAGE_ROOT=${IMAGE_ROOT:-"/public/home/jianghaitenggroup/jianglab/WorkSpace/MASS-SS3_Wave_224_448_V2/"}
BATCH_SIZE=${BATCH_SIZE:-128}
# 基础输出（eval.py 会自动加 _{ckpt_tag} 后缀）
OUTPUT_JSONL=${OUTPUT_JSONL:-"${BASE_DIR}/saved/val/results.jsonl"}
OUTPUT_XLSX=${OUTPUT_XLSX:-"${BASE_DIR}/saved/val/metrics.xlsx"}

cd "$BASE_DIR"

CHECKPOINT_ROOT="${BASE_DIR}/saved/train"

echo "[INFO] 开始遍历: $CHECKPOINT_ROOT"
found_any=false
for CKPT_DIR in "${CHECKPOINT_ROOT}"/*checkpoint*; do
  if [[ -d "$CKPT_DIR" ]]; then
    found_any=true
    CKPT_TAG=$(basename "$CKPT_DIR")
    echo "[INFO] 评估 checkpoint: $CKPT_TAG -> $CKPT_DIR"
    python ./eval.py \
      --ckpt_dir "$CKPT_DIR" \
      --system_prompt "$SYSTEM_PROMPT" \
      --subjects_json "$SUBJECTS_JSON" \
      --subjects_split "val" \
      --img_dir "$IMAGE_ROOT" \
      --batch_size "$BATCH_SIZE" \
      --output_jsonl "$OUTPUT_JSONL" \
      --output_xlsx "$OUTPUT_XLSX"
    echo "[INFO] 完成: $CKPT_TAG"
  fi
done

if ! $found_any; then
  echo "[WARN] 未找到包含 'checkpoint' 的目录: ${CHECKPOINT_ROOT}" >&2
  exit 1
fi

echo "[INFO] 所有 checkpoint 验证评估完成。输出目录: $(dirname "$OUTPUT_JSONL")"
